 /*
 *@(#)PageAnalyzer.java		 2000
 *
 * Copyright: MetaPlus Inc & Inquiry Management Systems Ltd.  2000
 *
 * This software is the proprietary information of MetaPlus Inc &
 * Inquiry Management Systems Ltd.
 * Use is subject to license terms.
 */
package com.ims.iaa.html;

import java.io.*;
import java.net.*;
import com.ims.iaa.*;
import com.ims.iaa.JavaScript.JavaScriptParser;

import java.util.HashMap;
import java.util.List;

import javax.swing.text.html.*;

/**
 * This class analyse a single HTML page.
 *
 * @author Michael Liao
 * @author Anthony Zhang
 * @author ShengMeng Bai
 *
 * @version 1.00, 25 Sept. 2000
 * @version 2.00, 23 JUL. 2004
 *
 */
//public class PageAnalyzer {
public class PageAnalyzer implements Runnable{
    //++changsheng 2006-11-6,because the spider may sink in dead status
    public void run(){
        setActiveTime();
        parse() ;
    }
    
    long lastActiveTime = 0 ;
    private void setActiveTime()  {
		lastActiveTime = System.currentTimeMillis();
	}
    
    //++Daniel 11-10-2006 For maxImpressions Limit
    /*
     * Returns the number of impressions encountered so far by this parser.
     */
    public int getNumImpressions() {
    	return m_iImageCount + m_iFlashCount + m_iPopupCount;
    }//end getNumImpressions
    
    public boolean checkStatus(){
        if(lastActiveTime == 0 ){
            return true;
        }else{
            if(System.currentTimeMillis() - lastActiveTime > IaaDebug.pageAnalyzeTimeOut ){
                m_ImpAnalyzer.analyzeOver();
                return false ;
            }else{
                return true ;
            }
        }
    }
    //--
    HTMLEditorKit kit = new HTMLEditorKit();

    private ImpAnalyzer m_ImpAnalyzer;
    private SpiderDataStream sStream;
    private URLConnection m_uc;
	/** 
	 * if current web page are popup window, 
	 * m_sParentPage is the parent page of this popup window,
	 * and you should call #SetParentPageURLStr() to set m_sParentPage.
	 */ 
	private String m_sParentPage = null;
	/*
	 * MAIN_PAGE, normal web page,
	 * POPUP_PAGE, some page send form normal page show through "window.open"
	 * SUB_PAGE, some page show in normal page show through "<frame> or <iframe>"
	 */
	public static final int MAIN_PAGE = 0;
	public static final int POPUP_PAGE = 1;
	public static final int SUB_PAGE = 2;
	private int m_iPageType;//MAIN_PAGE, POPUP_PAGE or SUB_PAGE
	private int m_iSubIndex = 0;
	private int m_iPopIndex = 0;
	
	private int m_iImageCount;
	private int m_iFlashCount;
	private int m_iPopupCount;

	private int specialJS = 0 ;
    public PageAnalyzer(ImpAnalyzer impAna, URLConnection URLc,int specialJS) {
    	this(impAna, URLc, MAIN_PAGE, null, 0);
    	this.specialJS = specialJS ;
    }

    public PageAnalyzer(ImpAnalyzer impAna, URLConnection URLc, int iPageType, String sParentURLStr, int iSubIndex) {
    	m_ImpAnalyzer = impAna;
    	sStream = m_ImpAnalyzer.getSpiderDataStream();
    	m_uc = URLc;
        m_iPageType = iPageType;
        m_sParentPage = sParentURLStr;
        m_iSubIndex = iSubIndex;
    }
  
    /*
     * because the parser has change, so this method has been modified.
     * old code is in the end of this method.
     */
    public void parse() {
        IaaDebug.begin("parse of pageAnalyzer " + m_uc.getURL());
        sStream.setPageBegin();

        setActiveTime();//changsheng 2006-11-6
        try {
    		StringBuffer sbHtml = getPageContent(m_uc);
    		  setActiveTime();//changsheng 2006-11-6
            IaaDebug.begin("parse html: " + m_uc.getURL());
            IaaUtil.log("parse html: " + m_uc.getURL());
            //pretreat the html
        	HTMLPretreatment preTreatement = preTreat(sbHtml, m_uc.getURL(), MAIN_PAGE);
        	//get the html after protreat
        	sbHtml = preTreatement.getDisposalHtml();
        	  setActiveTime();//changsheng 2006-11-6
        	try{
	        	//++parse the sub page
	        	parseSub(preTreatement, m_uc.getURL());
	        	parsePopup(preTreatement, m_uc.getURL());
	        	//--parse the sub page
        	}catch(Exception e){
        		IaaUtil.printStackTrace(e);
        	}
        	  setActiveTime();//changsheng 2006-11-6
        	ParserCallback objCallback = parseHtmlConetent(sbHtml, m_uc.getURL(), false);
        	  setActiveTime();//changsheng 2006-11-6
        	//set the analyzed over sign
       		m_ImpAnalyzer.analyzeOver();
       		
       		m_iImageCount += objCallback.getImageLinkCount();
       		m_iFlashCount += objCallback.getFlashLinkCount();
       		m_iPopupCount += objCallback.getPopupCount();
            sStream.println("\tIMG Link number: " + m_iImageCount);
            sStream.println("\tFLASH Link number: " + m_iFlashCount);
            sStream.println("\tPopup number: " + m_iPopupCount);
            IaaDebug.end("parse html: " + m_uc.getURL());
        } catch (java.util.EmptyStackException e) {
            IaaUtil.log("EmptyStack " + e);
        	IaaUtil.printStackTrace(e);
        } catch (IOException e) {
            IaaUtil.log("IOException while get content form: " + m_uc.getURL().toExternalForm());
        } catch (Exception e) {
            IaaUtil.log("Error parse the page " + e);
        	IaaUtil.printStackTrace(e);
            //e.printStackTrace();
        }catch(Throwable e){//changsheng
            
        }finally{
            m_ImpAnalyzer.analyzeOver();
        }

        sStream.setPageEnd();
        IaaDebug.end("parse of pageAnalyzer " + m_uc.getURL());
    }
    /**
     * Get the hmtl content form URLConnection object,
     * @param uc
     * @return if successful, return the html content, else return null.
     */
    private StringBuffer getPageContent(URLConnection uc) throws Exception
    {
		StringBuffer sbHtml = new StringBuffer();
        IaaUtil.log("Start pre-read html: " + uc.getURL());
        // read html content to StringBuffer.
        Reader preReader = new InputStreamReader(uc.getInputStream());
        int bufSize = 51200;//50k
        char[] buf = new char[bufSize];
		while(true){
            int moved = preReader.read(buf);
            if ( moved < 0 ) break;
            sbHtml.append(buf, 0, moved);
		}
		preReader.close();
		IaaUtil.log("End pre-read html: " + uc.getURL());
		return sbHtml;
    }
    /**
     * pretreat the html content, get content form javascript, get the source form frame and iframe tag
     * @param sbHtml, the html content
     * @param iPageType, MAIN_PAGE, POPUP_PAGE or SUB_PAGE
     * @return
     */
    private HTMLPretreatment preTreat(StringBuffer sbHtml, URL url, int iPageType)
    {
    	//Only for test, save the html to compare
    	if(IaaDebug.AuxFileDebug){
    		SavePage(sbHtml, ".html", iPageType);
    	}
    	//pretreat the html.
    	if(iPageType == MAIN_PAGE){
    		IaaUtil.log("Start pretreat html: " + url);
    	}
    	HTMLPretreatment preTreatment = new HTMLPretreatment(sbHtml, url, m_ImpAnalyzer,specialJS);
    	
    	sbHtml = preTreatment.getDisposalHtml();
    	
    	//Only for test, save the html to compare
    	if(IaaDebug.AuxFileDebug){
    		SavePage(sbHtml, "_treat.html", iPageType);
    	}
    	if(iPageType == MAIN_PAGE){
    		IaaUtil.log("End pretreat html: " + url);
    	}
    	
    	return preTreatment;
    }
    /**
     * Base on html content parse it, then return ParserCallback object to get some parse result.
     * @param sbHtml, html page content
     * @param url, page URL
     * @param bIsPopup, is popup document or not
     * @return
     */
    private ParserCallback parseHtmlConetent(StringBuffer sbHtml, URL url, boolean bIsPopup)
    {
		
        // Create a reader on the HTML content.
        Reader rd = new CharArrayReader(sbHtml.toString().toCharArray());
		
    	HTMLEditorKit.Parser parse = new HTMLParse().getParser();
    	ParserCallback objCallback = new ParserCallback(m_ImpAnalyzer, m_uc.getURL(), url, sbHtml);
       	//handle pop-up advertisement.
        if(bIsPopup){
        	objCallback.SetPopupParentPageURLStr(m_uc.getURL().toString());
            if(!objCallback.isValidHtml()){
            	return objCallback;
            }
        }
        try{
        	parse.parse(rd, objCallback, true);
        }catch (Exception e){
        	IaaUtil.printStackTrace(e);
        }
    	return objCallback;
    }
    private void SavePage(StringBuffer sbHtml, String sExtend, int iPageType)
    {
    	try{
    		String sFileName = escapeURLString(m_uc.getURL());
			if(iPageType == SUB_PAGE){
				sFileName += "Sub" +  m_iSubIndex;
			}else if(iPageType == POPUP_PAGE){
				sFileName += "Pop" +  m_iPopIndex;
    		}
    		PrintStream htmlOut = new PrintStream(
    				new FileOutputStream(IaaDebug.LogDirectory + "\\" +  sFileName + sExtend));
    		htmlOut.println(sbHtml);
    		htmlOut.close();
    	}catch(Exception e){
    		
    	}
    }
    public static String escapeURLString(URL url){
        String sFileName = url.toExternalForm();
		sFileName = sFileName.replaceAll("http://", "");
		sFileName = sFileName.replaceAll("\\\\", "_");
		sFileName = sFileName.replaceAll("/", "_");
		StringBuffer sbFile = new StringBuffer(sFileName);
		int iIndex = sbFile.indexOf("?");
		while(iIndex != -1){
		    sbFile.setCharAt(iIndex, '_');
		    iIndex = sbFile.indexOf("?", iIndex);
		}
		iIndex = sbFile.indexOf("&");
		while(iIndex != -1){
		    sbFile.setCharAt(iIndex, '_');
		    iIndex = sbFile.indexOf("&", iIndex);
		}
        
        return sbFile.toString();
    }
    //parse the sub page, which show in <frame> and <iframe> tag
    private void parseSub(HTMLPretreatment preTreatement, URL parentURL){
    	List lstSub = preTreatement.getSubPage();
    	if(lstSub != null){
    		for(int i = 0; i < lstSub.size(); i++){
    			m_iSubIndex = i;
    			try{
        			String sURL = (String)lstSub.get(i);
    				URL url = URLUtil.getURL(parentURL, sURL);
    				//if the URL is not accessible, return;
    				if(m_ImpAnalyzer.isDisallow(url)){
    					IaaUtil.log("Obey robots.txt: " + url);
    				    continue;
    				}
    				URLConnection uc = url.openConnection();
    				//++add by ShengMengBai at 2005-04-04
    				uc.setRequestProperty("User-Agent", m_ImpAnalyzer.getUserAgent());
    				//--add by ShengMengBai at 2005-04-04
//++changsheng for persisent connection
    	    		uc.setRequestProperty("Connection",   "Keep-Alive");
//--
    				//get the html content
    				StringBuffer sbHtml = getPageContent(uc);
    	            //pretreat the html
    	        	HTMLPretreatment preSub = preTreat(sbHtml, url, SUB_PAGE);
    	        	//get the html after protreat
    	        	sbHtml = preSub.getDisposalHtml();
    	    		//parse the content.
    	        	ParserCallback objCallback = parseHtmlConetent(sbHtml, url, false);
    	        	
    	       		m_iImageCount += objCallback.getImageLinkCount();
    	       		m_iFlashCount += objCallback.getFlashLinkCount();
    	        	
    	       		//if the sub page contain sub-sub page, deal with it.
    	        	try{
    		        	//++parse the sub page
    		        	parseSub(preSub, url);
    		        	//parsePopup(preSub, url);
    		        	//--parse the sub page
    	        	}catch(Exception e){
    	        		IaaUtil.printStackTrace(e);
    	        	}
    			}catch(Exception e){
    				IaaUtil.printStackTrace(e);
    			}
    		}
    	}
    }
    //parse the popup page, which show through javascript "window.open"
    private void parsePopup(HTMLPretreatment preTreatement, URL parentURL){
		HashMap map = preTreatement.getPopupPage();
		if(map != null){
			List lstURL = (List)map.get(JavaScriptParser.MAP_POPUP_URL);
			List lstCnt = (List)map.get(JavaScriptParser.MAP_POPUP_CONTENT);
			if(lstURL != null){
				for(int i = 0; i < lstURL.size(); i++){
	    			try{
		    			String sURL = (String)lstURL.get(i);
	    				URL url;
	    				StringBuffer sbHtml = null;
	    				//if the URL is blank, get the content form document content.
	    				if(sURL.equals("")){
	    					String sCnt = (String)lstCnt.get(i);
	    					if(sCnt != null && sCnt.equals(""))
	    					{
		    					url = parentURL;
			    				sbHtml = new StringBuffer(sCnt);
	    					}else{
	    					    continue;
	    					}
	    				}else{//if has URL, get content form URL.
		    				url = URLUtil.getURL(parentURL, sURL);
		    				//if the URL is not accessible, return;
		    				if(m_ImpAnalyzer.isDisallow(url)){
		    					IaaUtil.log("Obey robots.txt: " + url);
		    					continue;
		    				}
		    				URLConnection uc = url.openConnection();
		    				//++add by ShengMengBai at 2005-04-04
		    				uc.setRequestProperty("User-Agent", m_ImpAnalyzer.getUserAgent());
		    				//--add by ShengMengBai at 2005-04-04
//++changsheng for persisent connection
		    	    		uc.setRequestProperty("Connection",   "Keep-Alive");
//--
		    				//get the html content
		    				sbHtml = getPageContent(uc);
	    				}
	    	            //pretreat the html
	    		    	//Only for test, save the html to compare
	    		    	if(IaaDebug.AuxFileDebug){
	    		    		SavePage(sbHtml, ".html", POPUP_PAGE);
	    		    	}
	    		    	preTreatement.pretreatPopup(sbHtml, url, i);
	    	        	//get the html after protreat
	    	        	sbHtml = preTreatement.getDisposalHtml();
	    		    	if(IaaDebug.AuxFileDebug){
	    		    		SavePage(sbHtml, "_treat.html", POPUP_PAGE);
	    		    	}
	    	    		
	    	    		//parse the content.
	    	        	ParserCallback objCallback = parseHtmlConetent(sbHtml, url, true);
	    	        	
	    	       		m_iImageCount += objCallback.getImageLinkCount();
	    	       		m_iFlashCount += objCallback.getFlashLinkCount();
	    			}catch(Exception e){
	    				IaaUtil.printStackTrace(e);
	    			}
					m_iPopIndex++;
				}
			}
		}
    }

    /*
     *
      public static void main(String[] args) {
     String urlStr = args[0];
     //String urlStr = "http://www.zdnet.com";
     //String urlStr = "http://localhost:8100/ad2.html";
     PageAnalyzer pageAnalyzer = new PageAnalyzer();
     InputStream in= null;
     try  {
      URL url = URLUtil.getURL( urlStr );
      URLConnection uc = url.openConnection();
      in = uc.getInputStream();
      PrintStream out = new PrintStream( new FileOutputStream("pageResult.html") );
      out.println("<html><body>");
      out.print("<br><table width=100%><tr bgcolor=#d0d0d0><td>");
      out.print("<a href=\"" + uc.getURL().toExternalForm() + "\" target=\"_blank\">Page URL</a>");
      out.println("</td></tr>");
      out.println("<tr><td>");
      pageAnalyzer.parse( in, urlStr, out );
      out.println("</td></tr></table>");
      out.println("</body></html>");
      in.close();
      out.close();
     } catch ( MalformedURLException e ) {
      e.printStackTrace();
     } catch (IOException e ) {
      e.printStackTrace();
     }
        System.exit(0);
        }*/
}